package com.mine.crawler.config;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;


import com.mine.crawler.util.ImprovedLineBufferedReader;
import com.mine.crawler.util.PrintClass;


public class DOMZFileURLGenerator
{
	private static Logger log = Logger.getLogger(DOMZFileURLGenerator.class);
	
	private static String DMOZURL_filePath = InitializeCrawlerApplication.InitConfig.getProperty("DMOZ_Directory_Path") 
		+System.getProperty("file.separator") + Constants.DMOZ_Content_Path;
		
	public static void URLFileReader()
	{
		int count = 0;
		int Urlcount = 0;
		
		FileInputStream fis;
		byte[] readBuffer = new byte[Constants.BUFFERSIZE_2_MEGABYTE];
		List<String> lineListFromFile = new ArrayList<String> (Constants.LINES_READ_FROM_FILE);
		Map<Integer, String> HttpURLs = new HashMap<Integer, String>(Constants.URL_INITIAL_HASHMAP_SIZE);
		
		long startTime = System.currentTimeMillis();

		try
		{
			log.info("Started reading file");
			//fis= new FileInputStream (DMOZURL_filePath);
			//while(fis.)
			//fis.r
			FileReader fr = new FileReader(DMOZURL_filePath);
		    ImprovedLineBufferedReader br = new ImprovedLineBufferedReader(fr);
		    Iterator<String> lineListIterator;
		    String tempStr = "";
		    Integer tempInt = Integer.valueOf(0);
		    
		    while(br.FileReadComplete == false)	
		    {
		    	count++;
		    	lineListFromFile = br.readMultipleHTTPLines(Constants.LINES_READ_FROM_FILE);
		    	lineListIterator = lineListFromFile.iterator();
		    	//System.out.println("Size of arrayList ="+lineListFromFile.size());
		    	Urlcount = Urlcount + lineListFromFile.size();
		    	
		    	// TO STOP PREMATURALY
		    	if (count == 5)
		    		break;
		    	while(lineListIterator.hasNext())
		    	{
		    		tempStr = lineListIterator.next();
		    		tempInt = Integer.valueOf(tempStr.hashCode());
		    		HttpURLs.put(tempInt,tempStr );
		    		tempStr = null;
		    		tempInt = null;
		    	}
		    }
		    System.out.println("\n Time taken to read "+ count+" line file = "+ 
		    		Long.toString(System.currentTimeMillis()-startTime)	+ " ms");
		    log.info("Size of HTTP map should be = "+Urlcount+ " for file size "+count);
		    //System.out.println("Hash map " + HttpURLs);
		    //PrintClass.printHashMap(HttpURLs, Constants.LOGGER_LOCATION);
		    //8883506
		}
		catch(FileNotFoundException e)
		{
			System.out.println("Exception : "+e);
			e.printStackTrace();
		}
		catch(IOException e)
		{
			System.out.println("Exception : "+e);
			e.printStackTrace();
		}
		
	}
	
}
